#### 04_prepareRollCallBrazil-2.R ####

rm(list=ls())

print (version)

# platform       aarch64-apple-darwin20      
# arch           aarch64                     
# os             darwin20                    
# system         aarch64, darwin20           
# status                                     
# major          4                           
# minor          1.2                         
# year           2021                        
# month          11                          
# day            01                          
# svn rev        81115                       
# language       R                           
# version.string R version 4.1.2 (2021-11-01)
# nickname       Bird Hippie            

# Load the following libraries
library (openxlsx)
library (reshape2)
library (gtools)
library (car)
library (plyr)
library (MCMCpack)
library (zoo)
library (wnominate)
library (pscl)
library (runjags)
library (random)

# This file takes information on roll-call votes, 
# vote characteristics, and legislator characteristics
# and produces 10 long-format versions of the data.

# Set working directory
savePathRep <- "/all_data/"
setwd (savePathRep)

## set seed
set.seed (10882)

## Sample 20 rows from PTIRT
## (below we estimate models 10 different times,
## to allow uncertainty from PTIRT model
## to propagate through 1MPE and 2MPE models;
## we use the first 10 samples from the 
## following set of 20 random samples)
pt.draw <- sort (sample (1:400, 20))
pmdb.draw <- sort (sample (1:400, 20))
psdb.draw <- sort (sample (1:400, 20))
dem.draw <- sort (sample (1:400, 20))
pp.draw <- sort (sample (1:400, 20))
pr.draw <- sort (sample (1:400, 20))


# Prepare 10 different versions of the data
# that we will use for the purpose of fitting the actual party effects
# models 1MPE and 2MPE
for (h in 1:10)  {

    load ("prepared_roll_call_data_brazil.Rdata")
  
PT.pressure.index <- X.pt[pt.draw[h],]
PR.pressure.index <- X.pr[pr.draw[h],]
PP.pressure.index <- X.pp[pp.draw[h],]
PSDB.pressure.index <- X.psdb[psdb.draw[h],]
PMDB.pressure.index <- X.pmdb[pmdb.draw[h],]
DEM.pressure.index <- X.dem[dem.draw[h],]


# Get rid of votes that we deemed invalid in the party pressure exercise.
# These are in vectors which.vote and valid.vote
valid.party <- paste0 ("Sigla_Partido.", valid.vote)

w.party.54 <- data.frame (cbind (w.party.54$Nome_Deputado, w.party.54[,valid.party]))
colnames (w.party.54)[grep("w.party.54.Nome_Deputado", colnames(w.party.54))] <- "Nome_Deputado"
correct.w.party.54 <- correct.w.party.54[,valid.party]


# Do the same for the rc datasets
w.vote.54 <- data.frame (cbind (w.vote.54$Nome_Deputado, w.vote.54[,which.vote]))
colnames (w.vote.54)[grep("w.vote.54.Nome_Deputado", colnames(w.vote.54))] <- "Nome_Deputado"

is.element (which.vote, colnames (rc)) # All true
is.element (colnames (rc), which.vote) # some false
rc <- rc[,which.vote]

# Now we can build objects to be vectorized
names.only <- data.frame (Nome.Deputado=as.character(w.party.54$Nome_Deputado))
names.only$original.order <- c(1:nrow(names.only))

# Eliminate duplicate entries from deputy.Info
deputy.Info <- deputy.Info[duplicated (deputy.Info$RollcallName)==FALSE,]

deputy.features <- merge (x=names.only, y=deputy.Info
                          , by.x="Nome.Deputado", by.y="RollcallName"
                          , all.x=TRUE, all.Y=FALSE)
deputy.features <- deputy.features[order(deputy.features$original.order),]

# Merge deputy.features into w.party.54, excluding additional legislators that we do not have
deputy.features.correct <- merge (x=w.party.54, y=deputy.features
                                  , by.x="Nome_Deputado", by.y="Nome.Deputado"
                                  , all.x=TRUE)
deputy.features.correct <- deputy.features.correct[order(deputy.features.correct$original.order),]

w.party.54$original.order <- c(1:nrow(w.party.54))


# We need to melt rc and correct.w.party.54 into the right vector form
leg.index <- c(1:nrow(w.party.54))
roll.call <- as.data.frame (rc)
roll.call$leg <- leg.index
roll.call$leg.name <- w.party.54$Nome_Deputado
party.member <- as.data.frame (correct.w.party.54)
party.member$leg <- leg.index
party.member$leg.name <- w.party.54$Nome_Deputado

Votacoes.54$session <- Votes.54$session
vote.dates <- unlist (strsplit (as.character (Votacoes.54$Data[is.element(Votacoes.54$Id_Votacao, valid.vote)]), split=" "))[odd(1:708)]
vote.sessions <- Votacoes.54$session[is.element(Votacoes.54$Id_Votacao, valid.vote)]

colnames.vote.dates <- c()
for (i in 1:length(vote.dates)){
  colnames.vote.dates[i] <- paste (vote.dates[i], i, sep=".")
}

colnames.sessions <- c()
for (i in 1:length(vote.sessions)){
   colnames.sessions[i] <- paste (vote.sessions[i], i, sep=".")
}

date.roll.call <- roll.call
colnames (date.roll.call) <- c (colnames.vote.dates, "leg", "leg.name")

session.roll.call <- roll.call
colnames (session.roll.call) <- c (colnames.sessions, "leg", "leg.name")

molten.date.roll.call <- reshape2::melt(date.roll.call, id.vars=c("leg","leg.name"), variable.name="date", value.name="rc")
molten.session.roll.call <- reshape2::melt(session.roll.call, id.vars=c("leg","leg.name"), variable.name="session", value.name="rc")

identical (molten.date.roll.call$leg.name, molten.session.roll.call$leg.name)
molten.date.roll.call$session <- molten.session.roll.call$session
rm (molten.session.roll.call)

# Vectorize roll call
molten.roll.call <- reshape2::melt(roll.call, id.vars=c("leg","leg.name"), variable.name="vote", value.name="rc")
molten.roll.call$original.order <- 1:nrow(molten.roll.call)
molten.roll.call$vote.number <- substr(as.character(molten.roll.call$vote), start=6, stop=12)
levels (molten.roll.call$vote) <- as.numeric (c(1:length(levels(molten.roll.call$vote))))

identical (molten.roll.call$leg.name, molten.date.roll.call$leg.name) # should be true
molten.roll.call$correct.date <- gsub("\\..*","", molten.date.roll.call$date)
molten.roll.call$id <- paste (molten.roll.call$leg.name, molten.roll.call$correct.date, sep="-")


# Vectorize party membership
molten.party.member <- reshape2::melt(party.member, id.vars=c("leg", "leg.name"), variable.name="vote", value.name="party")
molten.party.member$original.order <- 1:nrow(molten.party.member)
molten.party.member$vote.no <- substr(molten.party.member$vote, start=15, stop=21)
molten.party.member$obs.id <- paste (molten.party.member$leg, molten.party.member$party, molten.party.member$vote.no, sep="-")
molten.party.member$leg.party.id <- paste (molten.party.member$leg, molten.party.member$party, sep="-")

non.vote.vars <- colnames (deputy.features.correct)[-grep("Sigla_Partido", colnames (deputy.features.correct))]
molten.deputy.features <- reshape2::melt(deputy.features.correct, id.vars=non.vote.vars, variable.name="vote", value.name="party.2")
molten.deputy.features$original.order <- 1:nrow(molten.deputy.features)
identical (molten.party.member$leg.name, molten.deputy.features$Nome_Deputado) # Should be TRUE
identical (molten.party.member$leg.name, molten.date.roll.call$leg.name) # Should be TRUE


# Now we need to separate a legislator in different parties into multiple legislators
molten.party.member <- molten.party.member[with(molten.party.member, order(molten.party.member$leg, molten.party.member$party)), ]
molten.party.member$party.noNA <- car::recode (molten.party.member$party, "NA='NA'")
new.leg.no <- c()
start <- 0
for (i in 1:max(molten.party.member$leg)){
  which.leg <- molten.party.member[molten.party.member$leg==i,]
  new.leg.all <- c()
  for (j in 1:length (rle (which.leg$party.noNA)$lengths)){
    new.leg <- rep (start+1, rle (which.leg$party.noNA)$lengths[j])
    start <- start+1
    new.leg.all <- c(new.leg.all, new.leg)
  }
  new.leg.no <- c(new.leg.no, new.leg.all)
}
molten.party.member$new.leg.no <- new.leg.no
molten.party.member <- molten.party.member[with(molten.party.member, order(molten.party.member$original.order)),]

# A different way of obtaining legislator-party unique values is to look assign a different number to each unique element of leg.party.id
molten.party.member <- molten.party.member[with(molten.party.member, order(molten.party.member$leg.party.id)), ]
alt.leg.no <- c()
for (i in 1:length (unique (molten.party.member$leg.party.id))) {
  which.leg <- unique (molten.party.member$leg.party.id)[i]
  alt.leg.no <- c(alt.leg.no, rep (i, length (molten.party.member$leg.party.id[molten.party.member$leg.party.id==which.leg])))
}
molten.party.member$alt.leg.no <- alt.leg.no
molten.party.member <- molten.party.member[with(molten.party.member, order(molten.party.member$original.order)),]

# Check one-to-one correspondence between alt.leg.no and new.leg.no
length (unique (paste(molten.party.member$alt.leg.no, molten.party.member$new.leg.no, sep="-")))  # will be 984 if all worked out well
identical (molten.party.member$leg.name, molten.date.roll.call$leg.name) # Should be TRUE

# Check one-to-many correspondence between leg.name and new.leg.no
by (molten.party.member$new.leg.no, molten.party.member$leg.name, unique)  # some legislators should have more than one number
by (molten.party.member$leg.name, molten.party.member$new.leg.no, unique)  # but there should only be one number per new.leg.no here!

# This is how old legislator numbers are broken into new legislator numbers (actual names for 'leg' are in w.party.54$Nome_Deputado)
by (molten.party.member$new.leg.no, molten.party.member$leg, unique)
by (molten.party.member$alt.leg.no, molten.party.member$leg, unique)
# There are 984 legislator-party observations (but many of these should drop, because they do not have votes)
# There are 642 unique names


# We need to find irrelevant legislator-party combinations
# These are legislators that switch "out" of the legislature, so their "party membership" is NA

irrelevant.leg <- c()
for (i in 1:length (unique (molten.party.member$new.leg.no))) {
  which.leg <- unique (molten.party.member$new.leg.no)[i]
  irrelevant.leg[i] <- invalid (molten.party.member$party[molten.party.member$new.leg.no==which.leg])
}
irrelevant.leg <- ifelse (irrelevant.leg==TRUE, 1, 0)

irrelevant.votes <- c()
for (i in 1:length (unique (molten.party.member$new.leg.no))) {
  which.leg <- unique (molten.party.member$new.leg.no)[i]
  irrelevant.votes[i] <- invalid (molten.roll.call$rc[molten.party.member$new.leg.no==which.leg])
}
irrelevant.votes <- ifelse (irrelevant.votes==TRUE, 1, 0)


irrelevant.obs <- is.element (molten.party.member$new.leg.no, unique (molten.party.member$new.leg.no)[irrelevant.leg==1])

# We need to get rid of 40773 irrelevant observations in molten.party.member and in molten.roll.call
molten.roll.call <- molten.roll.call[irrelevant.obs==FALSE,]
molten.party.member <- molten.party.member[irrelevant.obs==FALSE,]
molten.deputy.features <- molten.deputy.features[irrelevant.obs==FALSE,]
molten.date.roll.call <- molten.date.roll.call[irrelevant.obs==FALSE,]

molten.roll.call$new.original.order <- c(1:nrow(molten.roll.call))
molten.party.member$new.original.order <- c(1:nrow(molten.party.member))
molten.deputy.features$new.original.order <- c(1:nrow(molten.deputy.features))
molten.date.roll.call$new.original.order <- c(1:nrow(molten.date.roll.call))


# Check that molten.roll.call and molten.party.member retain identical order
identical (molten.roll.call$original.order, molten.party.member$original.order) # Needs to return TRUE
identical (molten.roll.call$original.order, molten.deputy.features$original.order) # Needs to return TRUE

identical (molten.roll.call$new.original.order, molten.party.member$new.original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, molten.deputy.features$new.original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, molten.date.roll.call$new.original.order) # Needs to return TRUE

# We need to construct the final.leg.no, because new.leg.no is now discontinuous
molten.party.member <- molten.party.member[with(molten.party.member, order(molten.party.member$new.leg.no)),]
final.leg.no <- c()
for (i in 1:length (unique(molten.party.member$new.leg.no))){
  which.leg <- unique(molten.party.member$new.leg.no)[i]
  tmp <- rep (i, length (molten.party.member$new.leg.no[molten.party.member$new.leg.no==which.leg]))
  final.leg.no <- c (final.leg.no, tmp)
}
molten.party.member$final.leg.no <- final.leg.no
molten.party.member <- molten.party.member[with(molten.party.member, order(molten.party.member$original.order)),]

# Check again that molten.roll.call and molten.party.member retain identical order
identical (molten.roll.call$original.order, molten.party.member$original.order) # Needs to return TRUE
identical (molten.roll.call$original.order, molten.deputy.features$original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, molten.party.member$new.original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, molten.deputy.features$new.original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, molten.date.roll.call$new.original.order) # Needs to return TRUE

molten.date.roll.call$vote.number <- molten.roll.call$vote.number

# Check that there are 782 unique legislator-party observations
max (molten.party.member$final.leg.no) # returns 782

# Check that each legislator/party-level unit has at least one observed vote (0 or 1)
by (molten.roll.call$rc, molten.party.member$final.leg.no, unique)

# Final check: No remaining legislator should ONLY have abstentions
legs.without.votes <- c()
for (i in 1:max(molten.party.member$final.leg.no)) {
  legs.without.votes[i] <- invalid (molten.roll.call$rc[molten.party.member$final.leg.no==i])
}
sum (legs.without.votes) # This should be 0 

# Turn party membership information into a numeric code
molten.party.member$party.no <- car::recode (molten.party.member$party, "'PT'=1
                                             ; 'PMDB'=2
                                             ; 'PP'=3
                                             ; 'PR'=4
                                             ; 'PSDB'=5
                                             ; 'DEM'=6
                                             ; else=7")
table (molten.party.member$party.no, molten.party.member$party)

molten.date.roll.call$real.date    <- gsub ('\\..*', '', x=molten.date.roll.call$date)
molten.date.roll.call$real.session <- gsub ('\\..*', '', x=molten.date.roll.call$session)

molten.date.roll.call$id <- paste (as.character (molten.date.roll.call$leg.name)
                                   , molten.date.roll.call$real.date
                                   , molten.date.roll.call$real.session, sep="-")

by (as.numeric (molten.date.roll.call$real.session), molten.date.roll.call$vote.number, unique)

## REIMPORT THE DATASET THAT LOOKS CAREFULLY AT "F", AND CODES "PRESENT BUT DID NOT VOTE"
ausenteData <- read.table ("brazil_present_absent_information.csv", sep=",", header=T)
ausenteData$date <- as.character (ausenteData$date)

unique(ausenteData$name[!is.element (ausenteData$name, w.party.54$Nome_Deputado)])
unique(w.party.54$Nome_Deputado[!is.element (w.party.54$Nome_Deputado, ausenteData$name)])

ausente.day   <- unlist (strsplit (as.character (ausenteData$date), split="/"))[seq(1,length(ausenteData$date)*3,by=3)]
ausente.month <- unlist (strsplit (as.character (ausenteData$date), split="/"))[seq(2,length(ausenteData$date)*3,by=3)]
ausente.year  <- unlist (strsplit (as.character (ausenteData$date), split="/"))[seq(3,length(ausenteData$date)*3,by=3)]

ausente.day   <- as.numeric (ausente.day)
ausente.month <- as.numeric (ausente.month)
ausente.year  <- as.numeric (ausente.year)

ausente.Date <- paste (ausente.month, ausente.day, ausente.year, sep="/")
ausenteData$correct.date <- ausente.Date

ausenteData$status <- as.character (ausenteData$status)
ausenteData$status <- ifelse (ausenteData$status=="P-" | ausenteData$status=="P*", "P", ausenteData$status)

# Check that dates are correctly arranged, and that days included in ausente dataset are in the vote dataset
# and vice versa
unique(ausenteData$correct.date)[!is.element (unique(ausenteData$correct.date), unique(vote.dates))] # These are not problematic
unique(vote.dates)[!is.element (unique(vote.dates), unique (ausenteData$correct.date))]  # These are more problematic
# The previous five dates are dates for which the "present/absent" system did not record legislator availability


ausenteData$correct.date[ausenteData$session==314 & ausenteData$correct.date=="11/8/2011"] <- "11/9/2011"
ausenteData$correct.date[ausenteData$session==115 & ausenteData$correct.date=="5/14/2013"] <- "5/15/2013"
ausenteData$correct.date[ausenteData$session==120 & ausenteData$correct.date=="5/15/2013"] <- "5/16/2013"
ausenteData$correct.date[ausenteData$session==183 & ausenteData$correct.date=="6/25/2013"] <- "6/26/2013"
ausenteData$correct.date[ausenteData$session==309 & ausenteData$correct.date=="10/8/2013"] <- "10/9/2013"


ausenteData$id <- paste (as.character (ausenteData$name)
                         , ausenteData$correct.date
                         , as.character (ausenteData$session)
                         , sep="-")

length (unique(ausenteData$id)[!is.element (unique(ausenteData$id), unique(molten.date.roll.call$id))]) # These are not problematic
length (unique(molten.date.roll.call$id)[!is.element (unique(molten.date.roll.call$id), unique(ausenteData$id))]) # These are problematic

# The next two should be identical
length (unique(ausenteData$id)[is.element (unique(ausenteData$id), unique(molten.date.roll.call$id))])
length (unique(molten.date.roll.call$id)[is.element (unique(molten.date.roll.call$id), unique(ausenteData$id))])

# Finally, merge
molten.date.roll.call <- merge (x=molten.date.roll.call, y=ausenteData
                           , by="id", all.x=TRUE, all.y=FALSE)
molten.date.roll.call <- molten.date.roll.call[with(molten.date.roll.call, order(molten.date.roll.call$new.original.order)),]

# Some duplicates in Cheibub dataset (multiple votes per session)
sum (duplicated (molten.date.roll.call$id))
unique (molten.date.roll.call$real.session[duplicated (molten.date.roll.call$id)])
unique (molten.date.roll.call$real.date[duplicated (molten.date.roll.call$id)])

# But no duplicates in original Patrick dataset
sum (duplicated(ausenteData$id))


table (molten.roll.call$rc, molten.date.roll.call$status)

pressureMatrix <- rbind (PT.pressure.index, PMDB.pressure.index, PP.pressure.index, PR.pressure.index, PSDB.pressure.index, DEM.pressure.index)

##Vote Data
vote.code<- read.table("brazil_votes_codification.txt")

vote.number<- vote.code$vote.number
Year<- vote.code$year
pcb.request<- vote.code$pcb.request
president.position<- vote.code$president.position
Procedural.Vote<- vote.code$Procedural.Vote

PT.leader.request.procedure<- vote.code$PT.leader.request.procedure 
PTLeaderDecision<- vote.code$PTLeaderDecision
PT.hi.std.LeaderDecision<- vote.code$PT.hi.std.LeaderDecision

PSDB.leader.request.procedure<- vote.code$PSDB.leader.request.procedure
PSDBLeaderDecision<- vote.code$PSDBLeaderDecision
PSDB.hi.std.LeaderDecision<- vote.code$PSDB.hi.std.LeaderDecision

DEM.leader.request.procedure<- vote.code$DEM.leader.request.procedure
DEMLeaderDecision<- vote.code$DEMLeaderDecision
DEM.hi.std.LeaderDecision<- vote.code$DEM.hi.std.LeaderDecision

PMDB.leader.request.procedure<- vote.code$PMDB.leader.request.procedure
PMDBLeaderDecision<- vote.code$PMDBLeaderDecision
PMDB.hi.std.LeaderDecision<- vote.code$PMDB.hi.std.LeaderDecision

PR.leader.request.procedure<- vote.code$PR.leader.request.procedure
PRLeaderDecision<- vote.code$PRLeaderDecision
PR.hi.std.LeaderDecision<- vote.code$PR.hi.std.LeaderDecision

PP.leader.request.procedure<- vote.code$PP.leader.request.procedure
PPLeaderDecision<- vote.code$PPLeaderDecision
PP.hi.std.LeaderDecision<- vote.code$PP.hi.std.LeaderDecision



positionMatrix <- rbind(PTLeaderDecision
                        , PMDBLeaderDecision
                        , PPLeaderDecision
                        , PRLeaderDecision
                        , PSDBLeaderDecision
                        , DEMLeaderDecision)

forAgainstMatrix <- rbind(PT.hi.std.LeaderDecision
                          , PMDB.hi.std.LeaderDecision
                          , PP.hi.std.LeaderDecision
                          , PR.hi.std.LeaderDecision
                          , PSDB.hi.std.LeaderDecision
                          , DEM.hi.std.LeaderDecision)

female    <- molten.deputy.features$Female 
seniority <- molten.deputy.features$Seniority
committee.chair <- ifelse (molten.deputy.features$Length.of.Committee.chair==0, 0, 1)
minister <- molten.deputy.features$Ministers.in.federal.government/4
governor <- molten.deputy.features$Governor/4
mayor    <- molten.deputy.features$Mayor/4
distance <- molten.deputy.features$Distance/1000
district.income <- ifelse (molten.deputy.features$District_income < quantile(molten.deputy.features$District_income, prob=c(0.25)), 1, 0)
district.farmer <- ifelse (molten.deputy.features$District_farmer > quantile(molten.deputy.features$District_farmer, prob=c(0.75)), 1, 0) 
log.district.M <- log (molten.deputy.features$district.magnitude)
district.M <- (molten.deputy.features$district.magnitude)/20
district.M2 <- (molten.deputy.features$district.magnitude)*(molten.deputy.features$district.magnitude)/400
district.rank  <- molten.deputy.features$district.rank
percentile.rank <- pmin (molten.deputy.features$district.magnitude, molten.deputy.features$district.rank)/molten.deputy.features$district.magnitude
year <- as.numeric (substr (molten.party.member$vote.no, start=1, stop=4))-2010


vote.features <- data.frame (president.position=president.position
                             , pcb.request=pcb.request
                             , year=Year
                             , Procedural.Vote=Procedural.Vote
                             , PP.leader.request.procedure=PP.leader.request.procedure
                             , PR.leader.request.procedure=PR.leader.request.procedure
                             , PT.leader.request.procedure=PT.leader.request.procedure
                             , DEM.leader.request.procedure=DEM.leader.request.procedure
                             , PSDB.leader.request.procedure=PSDB.leader.request.procedure
                             , PMDB.leader.request.procedure=PMDB.leader.request.procedure
                             , vote.number=as.character(vote.number))
vote.features$vote.number <- as.character (vote.features$vote.number)

# Merge with molten.roll.call, so that we automatically obtain vectorization of vote.features
sum (!is.element (vote.features$vote.number, molten.roll.call$vote.number)) # should be 0
sum (!is.element (vote.features$vote.number, molten.roll.call$vote.number)) # should be 0

molten.roll.call <- merge (molten.roll.call, vote.features, by="vote.number", all.x=TRUE)
molten.roll.call <- molten.roll.call[with(molten.roll.call, order(molten.roll.call$original.order)),]

# Quickly check again that molten.roll.call and molten.party.member retain identical order
identical (molten.roll.call$original.order, molten.party.member$original.order) # Needs to return TRUE
identical (molten.roll.call$original.order, molten.deputy.features$original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, molten.party.member$new.original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, molten.deputy.features$new.original.order) # Needs to return TRUE


# Party dummies
PT.dummy   <- ifelse (molten.party.member$party=="PT", 1, 0)
PSDB.dummy <- ifelse (molten.party.member$party=="PSDB", 1, 0)
DEM.dummy  <- ifelse (molten.party.member$party=="DEM", 1, 0)
PMDB.dummy <- ifelse (molten.party.member$party=="PMDB", 1, 0)
PR.dummy   <- ifelse (molten.party.member$party=="PR", 1, 0)
PP.dummy   <- ifelse (molten.party.member$party=="PP", 1, 0)



# Vote and legislator characteristics need to be multiplied by party.position and by party.pressure
pressure.for <- pressureMatrix*positionMatrix
colnames (pressure.for) <- valid.vote
pressure.for <- as.data.frame (pressure.for)
pressure.for$party <- c("PT","PMDB","PP","PR","PSDB","DEM")

pressureVector <- reshape2::melt(pressure.for, id.vars="party", variable.name="vote", value.name="pressure")
pressureVector$id <- paste (pressureVector$party, pressureVector$vote, sep="-")
pressureVector <- pressureVector[,!is.element(colnames(pressureVector), c("party","vote"))]



# Vote and legislator characteristics need to be multiplied by party.position and by high.party.pressure
high.std.pressure.for <- pressureMatrix*forAgainstMatrix
colnames (high.std.pressure.for) <- valid.vote
high.std.pressure.for <- as.data.frame (high.std.pressure.for)
high.std.pressure.for$party <- c("PT","PMDB","PP","PR","PSDB","DEM")

high.std.pressureVector <- reshape2::melt(high.std.pressure.for, id.vars="party", variable.name="vote", value.name="pressure")
high.std.pressureVector$id <- paste (high.std.pressureVector$party, high.std.pressureVector$vote, sep="-")
high.std.pressureVector <- high.std.pressureVector[,!is.element(colnames(high.std.pressureVector), c("party","vote"))]




identifier <- data.frame (id=paste (molten.party.member$party, molten.party.member$vote.no, sep="-"))
identifier$id <- as.character (identifier$id)
identifier$original.order <- molten.party.member$original.order
identifier$new.original.order <- molten.party.member$new.original.order

sum(!is.element (identifier$id, pressureVector$id)) #should be 72486
sum(!is.element (pressureVector$id, identifier$id)) #should be 0
sum(!is.element (identifier$id, high.std.pressureVector$id)) #should be 72486
sum(!is.element (high.std.pressureVector$id, identifier$id)) #should be 0

Pressure <- merge (x=identifier, y=pressureVector
                   , by.x="id", by.y="id", all.x=TRUE, all.y=FALSE)
Pressure <- Pressure[with(Pressure, order(Pressure$original.order)),]

High.std.Pressure <- merge (x=identifier, y=high.std.pressureVector
                   , by.x="id", by.y="id", all.x=TRUE, all.y=FALSE)
High.std.Pressure <- High.std.Pressure[with(High.std.Pressure, order(High.std.Pressure$original.order)),]

# Quickly check again that molten.roll.call and molten.party.member retain identical order
identical (molten.roll.call$original.order, Pressure$original.order) # Needs to return TRUE
identical (molten.roll.call$original.order, molten.deputy.features$original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, Pressure$new.original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, molten.deputy.features$new.original.order) # Needs to return TRUE
identical (molten.roll.call$original.order, High.std.Pressure$original.order) # Needs to return TRUE
identical (molten.roll.call$original.order, molten.deputy.features$original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, High.std.Pressure$new.original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, molten.deputy.features$new.original.order) # Needs to return TRUE


sum(!is.element (molten.roll.call$vote.number, vote.features$vote.number)) #should be 0
sum(!is.element (vote.features$vote.number, molten.roll.call$vote.number)) #should be 0

Vote.Features <- merge (x=molten.roll.call, y=vote.features
                        , by.x="vote.number", by.y="vote.number"
                        , all.x=TRUE, all.y=FALSE)
Vote.Features <- Vote.Features[with(Vote.Features, order(Vote.Features$original.order)),]

# Quickly check again that molten.roll.call and molten.party.member retain identical order
identical (molten.roll.call$original.order, Pressure$original.order) # Needs to return TRUE
identical (molten.roll.call$original.order, molten.deputy.features$original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, Pressure$new.original.order) # Needs to return TRUE
identical (molten.roll.call$new.original.order, molten.deputy.features$new.original.order) # Needs to return TRUE
identical (Pressure$new.original.order, Vote.Features$new.original.order)
identical (Pressure$original.order, Vote.Features$original.order)

Vote.Features$year.1 <- ifelse (Vote.Features$year.x==1, 1, 0)
Vote.Features$year.2 <- ifelse (Vote.Features$year.x==2, 1, 0)
Vote.Features$year.3 <- ifelse (Vote.Features$year.x==3, 1, 0)
Vote.Features$year.4 <- ifelse (Vote.Features$year.x==4, 1, 0)

# Check that these three columns coincide
cbind (molten.party.member$party, molten.party.member$vote.no, identifier$id)

Pressure$pressure <- ifelse (is.na(Pressure$pressure), 0, Pressure$pressure)
High.std.Pressure$pressure <- ifelse (is.na(High.std.Pressure$pressure), 0, High.std.Pressure$pressure)

R2 <- seniority*Pressure$pressure
R3 <- committee.chair*Pressure$pressure
R4 <- minister*Pressure$pressure
R8 <- district.income*Pressure$pressure
R28 <- PT.dummy*Vote.Features$pcb.request.x*Pressure$pressure
R29 <- PSDB.dummy*Vote.Features$pcb.request.x*Pressure$pressure
R30 <- DEM.dummy*Vote.Features$pcb.request.x*Pressure$pressure
R31 <- PMDB.dummy*Vote.Features$pcb.request.x*Pressure$pressure
R32 <- PR.dummy*Vote.Features$pcb.request.x*Pressure$pressure
R33 <- PP.dummy*Vote.Features$pcb.request.x*Pressure$pressure
R40 <- PT.dummy*Pressure$pressure
R41 <- PSDB.dummy*Pressure$pressure
R42 <- DEM.dummy*Pressure$pressure
R43 <- PMDB.dummy*Pressure$pressure
R44 <- PR.dummy*Pressure$pressure
R45 <- PP.dummy*Pressure$pressure
R124 <- log.district.M  * Pressure$pressure
R138 <- percentile.rank  * Pressure$pressure
R150 <- molten.roll.call$year * Pressure$pressure
R151 <- molten.roll.call$president.position * Pressure$pressure


# Identifying votes
v1 <- as.numeric (unique (molten.roll.call$vote[molten.roll.call$vote.number=="2012056"]))
v2 <- as.numeric (unique (molten.roll.call$vote[molten.roll.call$vote.number=="2012058"]))
v3 <- as.numeric (unique (molten.roll.call$vote[molten.roll.call$vote.number=="2013092"]))
v4 <- as.numeric (unique (molten.roll.call$vote[molten.roll.call$vote.number=="2013070"]))
v5 <- as.numeric (unique (molten.roll.call$vote[molten.roll.call$vote.number=="2011004"]))
v6 <- as.numeric (unique (molten.roll.call$vote[molten.roll.call$vote.number=="2011070"]))
potential.fix.bill <- sort (c(v1,v2,v3,v4,v5,v6))

new.party.color <- as.numeric (unlist( by (molten.party.member$party.no, molten.party.member$final.leg.no, unique)))
new.party.color <- car::recode(new.party.color, " 1='red'
                               ; 2='purple'
                               ; 3='orange'
                               ; 4='yellow'
                               ; 5='black'
                               ; 6='blue'
                               ; 7='gray'")



# Other objects that need to be restructured
full.party <- molten.party.member$party.no
full.party.not.small.parties <- ifelse (full.party > 6, 7, full.party)
bill.no <- molten.roll.call$vote

positionMatrix.exp      <- rbind (positionMatrix, rep(0, ncol(positionMatrix)))
forAgainstMatrix.exp    <- rbind (forAgainstMatrix, rep(0, ncol(forAgainstMatrix)))

position.vec <-  position.vec.strict <- c()
for (i in 1:length(bill.no)){
  position.vec[i] <- positionMatrix.exp[full.party.not.small.parties[i],bill.no[i]]
  position.vec.strict[i] <- forAgainstMatrix.exp[full.party.not.small.parties[i],bill.no[i]]
}

# List of objects to keep
objects_to_keep <- c("Vote.Features",
                     "PT.dummy",
                     "PSDB.dummy",
                     "PMDB.dummy",
                     "DEM.dummy",
                     "PP.dummy",
                     "PR.dummy",
                     "Pressure",
                     "molten.roll.call",
                     "molten.party.member",
                     "molten.date.roll.call",
                     "R2",
                     "R3",
                     "R4",
                     "R8",
                     "R40",
                     "R41",
                     "R42",
                     "R43",
                     "R44",
                     "R45",
                     "R124",
                     "R138",
                     "R150",
                     "seniority",
                     "committee.chair",
                     "district.income",
                     "percentile.rank",
                     "log.district.M",
                     "minister",
                     "position.vec.strict",
                     "X.pt",
                     "X.pmdb",
                     "X.psdb",
                     "X.dem",
                     "X.pr",
                     "X.pp")


name <- paste0 ("brazil_rollcall_long_format_", h, ".Rdata")

# Uncomment next line to save files
# do.call("save", c(objects_to_keep, list(file = name)))
}
